{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5da1fc3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3871264723d34d879e1b99bf6b5b5ec4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Fetching 3 files:   0%|          | 0/3 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64ee7c75237b4db98f0517f37d4381ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "test.parquet:   0%|          | 0.00/123M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "59fca75d8ea3494a9fb24bed607c2d75",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "train.parquet:   0%|          | 0.00/1.37G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "859a5f048ece4ffabc1cd3fff7123380",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       ".gitattributes:   0%|          | 0.00/2.46k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'/home/mesolitica/stt/Nonspeech7k'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "\n",
    "snapshot_download(\n",
    "    repo_id=\"W4ng1204/Nonspeech7k\",\n",
    "    repo_type=\"dataset\",\n",
    "    local_dir = './Nonspeech7k'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7106f383",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf nonspeech-7k\n",
    "!mkdir nonspeech-7k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "5e31f56e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████| 6289/6289 [00:01<00:00, 4429.06it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import random\n",
    "import json\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "questions = [\n",
    "    'given the labels\\n{labels}\\n\\nwhat is the label for the audio',\n",
    "    'what is the label for audio\\n\\nthe labels: {labels}'\n",
    "]\n",
    "\n",
    "df = pd.read_parquet('Nonspeech7k/train.parquet')\n",
    "unique_labels = str(df['classname'].unique().tolist())\n",
    "\n",
    "train = []\n",
    "for i in tqdm(range(len(df))):\n",
    "    metadata = df.iloc[i].to_dict()\n",
    "    metadata.pop('audio')\n",
    "    audio_filename = os.path.join('nonspeech-7k', f'{i}.wav')\n",
    "    with open(audio_filename, 'wb') as fopen:\n",
    "        fopen.write(df['audio'].iloc[i]['bytes'])\n",
    "    \n",
    "    train.append({\n",
    "        'question': random.choice(questions).format(labels = unique_labels),\n",
    "        'answer': df['classname'].iloc[i],\n",
    "        'audio_filename': audio_filename,\n",
    "        'metadata': json.dumps(metadata)\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "19af9746",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████| 725/725 [00:00<00:00, 5029.23it/s]\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_parquet('Nonspeech7k/test.parquet')\n",
    "\n",
    "test = []\n",
    "for i in tqdm(range(len(df))):\n",
    "    metadata = df.iloc[i].to_dict()\n",
    "    metadata.pop('audio')\n",
    "    audio_filename = os.path.join('nonspeech-7k', f'test-{i}.wav')\n",
    "    with open(audio_filename, 'wb') as fopen:\n",
    "        fopen.write(df['audio'].iloc[i]['bytes'])\n",
    "    \n",
    "    test.append({\n",
    "        'question': random.choice(questions).format(labels = unique_labels),\n",
    "        'answer': df['classname'].iloc[i],\n",
    "        'audio_filename': audio_filename,\n",
    "        'metadata': json.dumps(metadata)\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "1d4d9b30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6289, 725)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train), len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d2664398",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import Dataset\n",
    "\n",
    "dataset = Dataset.from_list(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "ad40e86f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22d9ab25afa74825a7271ce45a17ec58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8bd58f1e1a5a404cbe86c5a94ea5505a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/7 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "490f0bfaccdf4a8092c3d6594772e40f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading...:   0%|          | 0.00/225k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4628baae0958482888edbe77e093a5ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/1.49k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/datasets/mesolitica/Zeroshot-Audio-Classification-Instructions/commit/46faec8dad174d5fb53d934e82ea2b422a466858', commit_message='Upload dataset', commit_description='', oid='46faec8dad174d5fb53d934e82ea2b422a466858', pr_url=None, repo_url=RepoUrl('https://huggingface.co/datasets/mesolitica/Zeroshot-Audio-Classification-Instructions', endpoint='https://huggingface.co', repo_type='dataset', repo_id='mesolitica/Zeroshot-Audio-Classification-Instructions'), pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.push_to_hub('mesolitica/Zeroshot-Audio-Classification-Instructions', split = 'nonspeech7k_train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "2ccc1550",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = Dataset.from_list(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "ac44cb92",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b895bfe6347f46c68ae32a9858ee2e8f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading the dataset shards:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f16b3b736d5496b9a45efc29bf35aaa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating parquet from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c78eecd02fa49d2a8c3ccf107413b64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading...:   0%|          | 0.00/31.1k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65643a0eb8ae478cb7ccb7442797b625",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md:   0%|          | 0.00/1.63k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/datasets/mesolitica/Zeroshot-Audio-Classification-Instructions/commit/c60cbe3003608dd520c54cc00a3b6d33d41d9a3b', commit_message='Upload dataset', commit_description='', oid='c60cbe3003608dd520c54cc00a3b6d33d41d9a3b', pr_url=None, repo_url=RepoUrl('https://huggingface.co/datasets/mesolitica/Zeroshot-Audio-Classification-Instructions', endpoint='https://huggingface.co', repo_type='dataset', repo_id='mesolitica/Zeroshot-Audio-Classification-Instructions'), pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.push_to_hub('mesolitica/Zeroshot-Audio-Classification-Instructions', split = 'nonspeech7k_test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "ac7bcb61",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7014"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from glob import glob\n",
    "\n",
    "audio_files = glob('nonspeech-7k/*.wav')\n",
    "len(audio_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "fc6f32d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "with zipfile.ZipFile('nonspeech-7k.zip', 'w', zipfile.ZIP_DEFLATED) as zipf:\n",
    "    for f in audio_files:\n",
    "        zipf.write(f, arcname=f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "7ba4249b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Uploading files using Xet Storage..\n",
      "Uploading...: 100%|████████████████████████| 1.18G/1.18G [00:25<00:00, 46.4MB/s]\n",
      "https://huggingface.co/datasets/mesolitica/Zeroshot-Audio-Classification-Instructions/blob/main/nonspeech-7k.zip\n"
     ]
    }
   ],
   "source": [
    "!huggingface-cli upload mesolitica/Zeroshot-Audio-Classification-Instructions nonspeech-7k.zip \\\n",
    "--repo-type=dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "c8e16749",
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm nonspeech-7k.zip"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
